# -*- coding: utf-8 -*-
'''
@author Giang NH
@update 27/7/2011
'''

import urllib, cStringIO as StringIO
from mechanize import Browser
import commonlib, time, re, workerpool
import traceback, datetime, os
from MongoDbLog import MongoDbLog
from MongoModel import MongoModel
from lxml import etree
from termcolor import cprint

LOG = MongoDbLog("CongDongMang", "debug")

MONGO_SERVER = 'beta.mana.vn'
MONGO_PORT = 27017
MAX_DUPLICATED = 30

PREFIX = '/uploads/cdm/'
DOWNLOAD_PATH = '/home/hoangnamhai/HarvestedData/congdongmang'

def browser(url, data={}):
    br = Browser()
    response = None
    if len(data) > 0:
        query = urllib.urlencode(data)
        response = br.open(url, query)
    else:
        response = br.open(url)
    return br, response

def buildTree(html):
    parser = etree.HTMLParser(encoding='utf-8')
    return etree.parse(StringIO.StringIO(html), parser)

def checkExistImage(url):
    localFileName = "{0}/{1}.jpg".format(DOWNLOAD_PATH, commonlib.getMd5FileName(url))
    if not os.path.isfile(localFileName):
        commonlib.downloadImage(url, DOWNLOAD_PATH, PREFIX)

class CongDongMang(MongoModel):
    
    catUrl = 'http://congdongmang.vn/vi/list/gimg/{catid}/link.aspx'
    categories = {108: 'cutegirl', 109: 'vuicuoi', 114: '18plus', 103: 'tonghop'}
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)

    def identifyId(self, link):
        pat = r'gimg/\w+/(\d+)'
        return commonlib.extractWithRegEx(pat, link, 1)

    def processCategory(self, catId=109):
        LOG.debug('start processCategory(catId={0}, catName={1})'.format(catId, self.categories[catId]))
        url = self.catUrl.format(catid=catId)
        data = {}
        currentDuplicated = 0
        try:
            db = self.connection['congdongmang']
            collection = db[self.categories[catId]]
            while currentDuplicated <= MAX_DUPLICATED:
                br, res = browser(url, data)
                br.select_form(name='aspnetForm')
                br.form.set_all_readonly(False)
                html = res.read()
                tree = buildTree(html)
                fbreak = False
                for item in tree.xpath("//li[@class='thumbs-video-wrapper']//h3/a"):
                    if currentDuplicated > MAX_DUPLICATED:
                        LOG.info("Trùng quá ngưỡng {0} cho phép".format(MAX_DUPLICATED))
                        cprint("Trùng quá ngưỡng {0} cho phép".format(MAX_DUPLICATED), 'red')
                        fbreak = True; break
                    link = commonlib.urlJoin(url, item.get('href', ''))
                    imageId = self.identifyId(link)
                    if imageId == '': continue
                    hashUrl = commonlib.getMD5Hash(imageId)
                    isExist = collection.find_one({'hashUrl': hashUrl})
                    if isExist:
                        currentDuplicated += 1
                        cprint('Already exist in database !!! ({0})'.format(link), 'green')
                        continue
                    self.processPage(catId, link, hashUrl)
                if fbreak: break
                nextNode = tree.xpath("//div[@class='_datapager']/span/span/following-sibling::*[1]")
                if len(nextNode) == 0: break
                if nextNode[0].get('disabled', '') != '': break
                '''javascript:__doPostBack('Pictures_all$lvList$dataPager$ctl00$ctl01','')'''
                href = nextNode[0].get('href', '')
                pat = r"\('(.+)',"
                href = commonlib.extractWithRegEx(pat, href, 1)
                data = {'__EVENTTARGET': '{0}'.format(href), 
                    '__EVENTARGUMENT': '', 
                    'ScriptManager1': 'tPictures_all$UpdatePanel2|{0}'.format(href), 
                    '__VIEWSTATE': '''{0}'''.format(br['__VIEWSTATE']),
                    '__EVENTVALIDATION': br['__EVENTVALIDATION']
                    }
                print '********************************************'
                print 'DEBUG: Next page: {0}'.format(nextNode[0].text)
                print '********************************************'
        except:
            LOG.error(traceback.format_exc())

    def processPage(self, catId, url, hashUrl):
        LOG.debug('start processPage(catId={0}, url={1})'.format(catId, url))
        try:
            db = self.connection['congdongmang']
            collection = db[self.categories[catId]]
            tree = commonlib.getXMLTree(url)
            if tree == None or tree == '': return
            imgNode = tree.xpath("//div[@id='meeboitem']//img[@id='imageContent']")
            if len(imgNode) > 0:
                link = commonlib.getAttribText(imgNode, 'src')
                link = commonlib.urlJoin(url, link) if link != '' else ''
                if link == '' or not re.search(r'jpg|gif|png|bmp|jpeg|jpe$', link): return
                dlink = commonlib.downloadImage(link, DOWNLOAD_PATH, PREFIX)
                if dlink == '': return
                comment = commonlib.getElementText(tree.xpath("//h1[@class='video-title float-left']"))
                timeUpload = commonlib.getElementText(tree.xpath("//span[@class='time-upload']"))
                
                collection.save({
                    'title': comment,
                    'rlink': link,
                    'link': dlink,
                    'timeupload': timeUpload,
                    'timestamp': time.time(), 
                    'lastupdate': datetime.datetime.now(),
                    'hashUrl': hashUrl,
                    'plink': url
                })
                cprint(link, 'yellow')
                print dlink
                cprint(comment, 'green')
                print timeUpload
        except:
            LOG.error(traceback.format_exc())
        finally:
            LOG.debug('finished processPage(catId={0}, url={1})'.format(catId, url))
    
    def process(self):
        try:
            pool = workerpool.WorkerPool(size=4)
            pool.map(self.processCategory, self.categories.keys())
            pool.shutdown()
            pool.wait()
        except:
            LOG.error(traceback.format_exc())
        return
    
def foreceQuit():
    pid = os.getpid()
    os._exit(1)
    os.kill(pid, 9)

if __name__ == '__main__':
    LOG.debug("start crawler congdongmang.vn")
    congdongmang = CongDongMang(MONGO_SERVER, MONGO_PORT)
#    congdongmang.processCategory()
    congdongmang.process()
    LOG.debug("finished crawler congdongmang.vn at {0}".format(datetime.datetime.now()))
    foreceQuit()
    
    
    